{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir('../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/njuciairs/wangshuai/test/FinancialNagetiveEntityJudge'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation.evaluate import evaluate\n",
    "from data_utils.basic_data import load_train_val_dataset,load_basic_dataset\n",
    "from results_process.regulizer import remove_nine,remove_short_entity\n",
    "from results_process.utils import load_model_rs\n",
    "from results_process.bert_entity_model import reduce_rs_by_id\n",
    "from functools import reduce\n",
    "import numpy as np\n",
    "from data_utils.bert_multi_class_data import get_train_val_data_loader, get_test_loader,TestEntityDataset\n",
    "import pandas as pd\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_df = load_model_rs(model_name='BertSentiEntity_cross',version_id=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#交叉训练模型1 multi_class_cross1  \n",
    "#交叉训练模型2 BertSentiEntity_cross"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = [load_model_rs(model_name='BertSentiEntity_cross',version_id=i) for i in range(1,10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "senti_dic = {}#id:senti\n",
    "entity_dic = {}#(id,entity):predict\n",
    "for df in dfs:\n",
    "    for id,negative,predict_list,entity_list in df.values:\n",
    "        predict_list = eval(predict_list)\n",
    "        entity_list = eval(entity_list)\n",
    "        if id not in senti_dic:\n",
    "            senti_dic[id] = []\n",
    "        senti_dic[id].append(negative)\n",
    "        for entity,predict in zip(entity_list,predict_list):\n",
    "            key = (id,entity)\n",
    "            if key not in entity_dic:\n",
    "                entity_dic[key] = []\n",
    "            entity_dic[key].append(predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(entity_dic[('2714c581', '联璧金融')]).most_common(1)[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_most(predict_list):\n",
    "    count = Counter(predict_list)\n",
    "    return count.most_common(1)[0][0]\n",
    "for k,v in senti_dic.items():\n",
    "    v = get_most(v)\n",
    "    senti_dic[k] = v\n",
    "for k,v in entity_dic.items():\n",
    "    v = get_most(v)\n",
    "    entity_dic[k] = v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_rs_dic = {}\n",
    "for (id,entity),predict in entity_dic.items():\n",
    "    if id not in entity_rs_dic:\n",
    "        entity_rs_dic[id] = []\n",
    "    if predict==1:\n",
    "        entity_rs_dic[id].append(entity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs = []\n",
    "for id,senti in senti_dic.items():\n",
    "    entities = entity_rs_dic[id]\n",
    "    if len(entities) == 0:\n",
    "        estr = np.nan\n",
    "    else:\n",
    "        estr = ';'.join(entities)\n",
    "    rs.append((id,senti,estr))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs_df = pd.DataFrame(rs,columns=['id','negative','key_entity'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#去重：把更短的去掉\n",
    "import numpy as np\n",
    "def remove_short_entity_by_long(entity_str):\n",
    "    \"\"\"\n",
    "    除去key_entity中同一实体的较短名称\n",
    "    :param entity_str:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    if not isinstance(entity_str, str):\n",
    "        return entity_str\n",
    "    entities = entity_str.split(';')\n",
    "    states = np.ones(len(entities))\n",
    "    for i, e in enumerate(entities):\n",
    "        for p in entities:\n",
    "            if e in p and len(e) < len(p):\n",
    "                print('removed %s by %s'%(e,p))\n",
    "                states[i] = 0\n",
    "    rs = []\n",
    "    for i, e in enumerate(entities):\n",
    "        if states[i] == 1:\n",
    "            rs.append(e)\n",
    "    rs = ';'.join(rs)\n",
    "    return rs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_trans_map():\n",
    "    from data_utils.basic_data import load_basic_dataset\n",
    "    train_df = load_basic_dataset('train')\n",
    "    srcs = train_df['entity'].map(lambda x :list(str(x).split(';')))\n",
    "    dests =  train_df['key_entity'].map(lambda x :list(str(x).split(';')))\n",
    "    trans_map = {}\n",
    "    for srcs,dests in list(zip(srcs,dests)):\n",
    "        for src in srcs:\n",
    "            if src == '':\n",
    "                continue\n",
    "            for e in srcs:\n",
    "                if e== '':\n",
    "                    continue\n",
    "                if (src in e or e in src) and e!=src:\n",
    "                    if src in dests:\n",
    "                        trans_map[src+'-'+e] = src\n",
    "                        trans_map[e+'-'+src] = src\n",
    "                    if e in dests:\n",
    "                        trans_map[src+'-'+e] = e\n",
    "                        trans_map[e+'-'+src] = e\n",
    "    return trans_map\n",
    "def trans_keys(trans_map,entity_str):\n",
    "    if not isinstance(entity_str,str):\n",
    "        return entity_str\n",
    "    es = list(filter(lambda x:str(x).strip()!='',entity_str.split(';')))\n",
    "    rs = set()\n",
    "    for e in es:\n",
    "        finded = False\n",
    "        for y in es:\n",
    "            if e+'-'+y in trans_map and e!=y:\n",
    "                rs.add(trans_map[e+'-'+y])\n",
    "                finded = True\n",
    "        if not finded:\n",
    "            rs.add(e)\n",
    "    if len(rs) > 0:\n",
    "        rs = ';'.join(list(rs))\n",
    "    else:\n",
    "        rs = np.nan\n",
    "    return rs\n",
    "trans_map = get_trans_map()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "removed 优库速购 by 深圳优库速购\n",
      "removed 瑞波 by 瑞波币\n",
      "removed 联璧 by 联璧金融\n",
      "removed 天天投 by 天天投金融\n",
      "removed 商银信支付 by 商银信支付服务有限公司\n",
      "removed 蜜蜂 by 蜜蜂财富\n",
      "removed 米融 by 易米融\n",
      "removed 东源 by 山东源大工贸\n",
      "removed 玖富 by 北京玖富集团\n",
      "removed 宝银创赢 by 上海宝银创赢投资管理有限公司\n",
      "removed 富通天下 by 深圳市富通天下投资管理有限公司\n",
      "removed 深圳市富通 by 深圳市富通天下投资管理有限公司\n",
      "removed 无忧借条 by ????无忧借条\n",
      "removed 优宝 by 优宝汇\n",
      "removed 壹佰金融 by 深圳壹佰金融\n",
      "removed 青岛九州商品交易中心 by 青岛九州商品交易中心有限公司\n",
      "removed 资易贷 by 资易贷北京金融信息服务有限公\n",
      "removed 雪橙 by 雪橙金服\n",
      "removed 凯福德 by KFD凯福德\n",
      "removed 仁和融兴 by 青岛仁和融兴投资有限公司\n",
      "removed 资易贷 by 资易贷(北京）金融信息有限公司公司\n",
      "removed Square by BlatentSquareripoff\n",
      "removed 澳瑞克 by OracleFX澳瑞克\n",
      "removed 中南大宗 by 中南大宗商品电子商务有限公司\n",
      "removed 惠金服 by 城惠金服\n",
      "removed 富民投资 by 富民投资网\n",
      "removed 小资钱包 by 小资钱包公司(资易贷平台)\n",
      "removed 兴业财富 by 北方兴业财富\n",
      "removed 余盆网 by 北京华澳融信（余盆网）\n",
      "removed 北京华澳融信 by 北京华澳融信（余盆网）\n",
      "removed 智云 by 智云金融\n",
      "removed 智融财富 by 深圳市智融财富电商投资有限公司\n",
      "removed 贵金属 by 贵金属投资\n",
      "removed 上海易贷网 by 上海易贷网金融信息服务有限公司\n",
      "removed 中吴财富 by 上海中吴财富投资管理集团有限公司\n",
      "removed 钱宝 by 钱宝财\n",
      "removed 节节贷 by 广西弘尚节节贷网络信息服务集团有限公司\n",
      "removed 北京华澳翼时代 by 北京华澳翼时代信息技术有限责任公司\n",
      "removed 北京华澳融信 by 北京华澳融信国际投资管理咨询有限公司\n",
      "removed 市大时代 by 深圳市大时代资产管理有限公司\n",
      "removed 日融财富 by 宁波日融财富投资管理有限公司\n",
      "removed 海象理财 by 北京海象理财\n",
      "removed 零钱罐 by 零钱罐APP\n",
      "removed 冠e通 by 冠e通平台\n",
      "removed 誉东方投资管理 by 深圳市誉东方投资管理平台\n",
      "removed 大宗商品交易 by 河北滨海大宗商品交易市场服务有限公司\n",
      "removed 大宗商品交易 by 河北滨海大宗商品交易市场\n",
      "removed 河北滨海大宗商品交易市场 by 河北滨海大宗商品交易市场服务有限公司\n",
      "removed 资易贷 by 海淀资易贷平台\n",
      "removed 两只老虎 by 两只老虎理财\n",
      "removed 老虎理财 by 两只老虎理财\n",
      "removed 海象理财 by 北京海象理财\n",
      "removed 联璧 by 联璧金融\n",
      "removed 钱海湾金融 by 钱海湾金融公司\n",
      "removed 渤海商品交易 by 天津渤海商品交易所\n",
      "removed 冠群驰骋 by ????冠群驰骋\n",
      "removed 冠群驰骋 by 冠群驰骋商务信息咨询（天津）有限公司\n",
      "removed 联金所 by 联金所康\n",
      "removed 米咖 by 米咖网\n",
      "removed 中吴财富 by 上海中吴财富投资管理集团有限公司\n",
      "removed 资易贷 by 资易贷（北京）金融信息服务有限公司旗\n",
      "removed 金易融 by 金易融（北京）网络科技有限公司\n",
      "removed 唐小僧 by 唐小僧之后\n",
      "removed 以太坊 by 以太坊ETH\n",
      "removed 翱晟投资 by 台州翱晟投资公司\n",
      "removed 老虎金融 by 老虎金融信息服务（北京）有限公司\n",
      "removed 冠群驰骋 by 冠群驰骋投资关联(北京)有限公司\n",
      "removed 冠群驰骋 by 冠群驰骋投资管理(北京)有限公司\n",
      "removed 中子星投资 by 中子星投资有限公司\n",
      "removed 钱宝 by 钱宝财\n",
      "removed 一号家居网 by 一号家居网装饰公司\n",
      "removed 上海文化产权交易所 by 上海文化产权交易所股份有限公司\n",
      "removed 仁和融兴 by 青岛仁和融兴投资有限公司\n",
      "removed 宜贷网 by ?宜贷网\n",
      "removed 海贷金服 by 海贷金服体验金\n",
      "removed 易商通 by 北京易商通科技有限公司\n",
      "removed 房融 by 房融所\n",
      "removed 钱宝 by 钱宝财\n",
      "removed 渤海创投 by ?渤海创投子公司智慧蜂巢\n",
      "removed 齐鲁商品 by 齐鲁商品交易中心\n",
      "removed 资易贷 by 资易贷（小资钱包）\n",
      "removed 小资钱包 by 资易贷（小资钱包）\n",
      "removed 山东海倍电子商务 by 山东海倍电子商务股份有限公司\n",
      "removed 山东海倍电子商务 by 山东海倍电子商务股份有限公司\n",
      "removed 浙商银行 by 浙商银行直销银行\n",
      "removed 渤海创投 by 渤海创投集团通\n",
      "removed 汇聚财富 by 上海汇聚财富投资\n",
      "removed 米融 by 易米融\n",
      "removed 御顺金融 by 成都御顺金融贷款公司\n",
      "removed 渤海商品交易所 by 天津渤海商品交易所\n",
      "removed 瑞财 by 恒瑞财富\n",
      "removed 中海投 by 中海投资\n",
      "removed 中海投 by 上海中海投资产管理有限公司\n",
      "removed 中海投资 by 上海中海投资产管理有限公司\n",
      "removed 阳光保险 by 阳光保险承保\n",
      "removed 日融财富 by 宁波日融财富投资管理有限公司\n",
      "removed 小资 by 小资钱包\n",
      "removed 富民投资 by 富民投资网\n",
      "removed 宜贷网 by 宜贷网（原易贷网）\n",
      "removed 天合联盟 by 安徽天合联盟科技有限公司\n",
      "removed 宜贷网 by 宜贷网（原易贷网）\n",
      "removed 钱宝 by 钱宝财\n",
      "removed 信e贷 by 久信e贷\n",
      "removed 易商通 by 北京易商通科技有限公司\n",
      "removed 易商通 by 北京易商通科技有限公司\n",
      "removed 深圳高新 by 深圳高新盛\n",
      "removed 高新盛 by 深圳高新盛\n",
      "removed 翱晟投资 by 台州翱晟投资公司\n",
      "removed 宜贷网 by ?宜贷网\n",
      "removed 高新盛 by 深圳高新盛创投电子商务有限公司\n",
      "removed 懒财主 by 前海懒财主金融\n",
      "removed 懒财主 by 海懒财主金融信息服务（深圳）有限公司\n",
      "removed 资易贷 by 北京资易贷公司（小资钱包)\n",
      "removed 小资钱包 by 北京资易贷公司（小资钱包)\n",
      "removed 银通投资 by 中润银通投资(北京)有限公司\n",
      "removed 乐贷 by 网乐贷\n",
      "removed 米融 by 易米融\n",
      "removed 资易贷 by 资易贷（北京）金融信息服务公司\n",
      "removed 三农金服 by 深圳三农金服\n",
      "removed 世纪贷 by 世纪贷互联网金融服务有限公司\n",
      "removed 聚财猫 by ????聚财猫\n",
      "removed 36氪 by 36氪股权众筹\n",
      "removed 国润 by 江西国润\n",
      "removed 亚太投资 by 北京亚太投资\n",
      "removed 恒宇 by 恒宇天泽\n",
      "removed 圣盈信 by 圣盈信CIFS\n",
      "removed 北京华澳融信 by 北京华澳融信国际投资管理咨询有限公司\n",
      "removed 汇金方格 by 汇金方格（北京）投资管理有限公司\n",
      "removed 融通资产 by 北京圆融通资产管理有限公司\n",
      "removed 盛付通 by 上海盛付通电子支付服务有限公司陕西分公司\n",
      "removed 粤融泰富 by 广东粤融泰富网络信息服务有限公司\n",
      "removed 大宗商品交易 by 河北滨海大宗商品交易市场\n",
      "removed 海钜信达 by 深圳市海钜信达投资发展有限公司\n",
      "removed 陆金所 by 西部陆金所\n"
     ]
    }
   ],
   "source": [
    "rs_df['key_entity'] = rs_df['key_entity'].map(lambda x: trans_keys(trans_map,x)).map(remove_short_entity_by_long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs_df.to_csv('evaluation/tmp/multi_choice_cross1-9_1024.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
